{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## todo: \n",
    "# 解决测评函数不能引入cv的问题 done\n",
    "# 建立cv流程进行调参 在模型处引入权重因素\n",
    "# 清理异常点(小于0) done\n",
    "\n",
    "#训练集的最优是过拟合的结果，选择次优方案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_test=pd.read_csv('input/train_time.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_test=train_test.fillna(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>college</th>\n",
       "      <th>rank</th>\n",
       "      <th>total_people</th>\n",
       "      <th>rank_percent</th>\n",
       "      <th>countM1</th>\n",
       "      <th>price_sumM1</th>\n",
       "      <th>price_avgM1</th>\n",
       "      <th>price_maxM1</th>\n",
       "      <th>...</th>\n",
       "      <th>地点263_avg</th>\n",
       "      <th>地点263_max</th>\n",
       "      <th>地点263_min</th>\n",
       "      <th>地点263_median</th>\n",
       "      <th>地点840_count</th>\n",
       "      <th>地点840_sum</th>\n",
       "      <th>地点840_avg</th>\n",
       "      <th>地点840_max</th>\n",
       "      <th>地点840_min</th>\n",
       "      <th>地点840_median</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2933.0</td>\n",
       "      <td>0.000341</td>\n",
       "      <td>49.0</td>\n",
       "      <td>201.31</td>\n",
       "      <td>4.108367</td>\n",
       "      <td>36.4</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2933.0</td>\n",
       "      <td>0.000682</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.00</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>6.157895</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1565.0</td>\n",
       "      <td>1572.0</td>\n",
       "      <td>0.995547</td>\n",
       "      <td>97.0</td>\n",
       "      <td>347.74</td>\n",
       "      <td>3.584948</td>\n",
       "      <td>10.0</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1570.0</td>\n",
       "      <td>1572.0</td>\n",
       "      <td>0.998728</td>\n",
       "      <td>98.0</td>\n",
       "      <td>491.01</td>\n",
       "      <td>5.010306</td>\n",
       "      <td>17.5</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2301.0</td>\n",
       "      <td>0.000435</td>\n",
       "      <td>27.0</td>\n",
       "      <td>82.96</td>\n",
       "      <td>3.072593</td>\n",
       "      <td>22.3</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.500000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 432 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label  college    rank  total_people  rank_percent  countM1  \\\n",
       "0   0    0.0      9.0     1.0        2933.0      0.000341     49.0   \n",
       "1   1    0.0      9.0     2.0        2933.0      0.000682     -1.0   \n",
       "2   8    0.0      6.0  1565.0        1572.0      0.995547     97.0   \n",
       "3   9    0.0      6.0  1570.0        1572.0      0.998728     98.0   \n",
       "4  10    0.0      3.0     1.0        2301.0      0.000435     27.0   \n",
       "\n",
       "   price_sumM1  price_avgM1  price_maxM1      ...       地点263_avg  地点263_max  \\\n",
       "0       201.31     4.108367         36.4      ...            -1.0       -1.0   \n",
       "1        -1.00    -1.000000         -1.0      ...            -1.0       -1.0   \n",
       "2       347.74     3.584948         10.0      ...            -1.0       -1.0   \n",
       "3       491.01     5.010306         17.5      ...            -1.0       -1.0   \n",
       "4        82.96     3.072593         22.3      ...            -1.0       -1.0   \n",
       "\n",
       "   地点263_min  地点263_median  地点840_count  地点840_sum  地点840_avg  地点840_max  \\\n",
       "0       -1.0          -1.0         -1.0       -1.0  -1.000000       -1.0   \n",
       "1       -1.0          -1.0         19.0      117.0   6.157895        7.0   \n",
       "2       -1.0          -1.0         -1.0       -1.0  -1.000000       -1.0   \n",
       "3       -1.0          -1.0         -1.0       -1.0  -1.000000       -1.0   \n",
       "4       -1.0          -1.0          2.0        7.0   3.500000        4.0   \n",
       "\n",
       "   地点840_min  地点840_median  \n",
       "0       -1.0          -1.0  \n",
       "1        4.0           6.0  \n",
       "2       -1.0          -1.0  \n",
       "3       -1.0          -1.0  \n",
       "4        3.0           3.5  \n",
       "\n",
       "[5 rows x 432 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## Evaluation \n",
    "def f1_macro(label_truth, predictions):\n",
    "    df=pd.DataFrame(columns=[\"subsidy_x\",\"subsidy_y\"])\n",
    "    df.subsidy_y=predictions\n",
    "    df.subsidy_x=np.array(label_truth)\n",
    "    df.subsidy_y = df.subsidy_y.apply(lambda x:int(x))\n",
    "\n",
    "    \n",
    "    correct = df[df['subsidy_x'] == df['subsidy_y']]\n",
    "    s = 0\n",
    "    for i in [1000, 1500, 2000]:\n",
    "        print '\\n%d'%i\n",
    "        if sum(df['subsidy_x'] == i)!=0:\n",
    "            r = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_x'] == i)\n",
    "            print 'Recall---%s'%r\n",
    "        else: \n",
    "            r=0\n",
    "        if sum(df['subsidy_y'] == i)!=0:\n",
    "            p = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_y'] == i)        \n",
    "            print 'Precision---%s'%p\n",
    "        else:\n",
    "            p=0\n",
    "        if (r+p)!=0:\n",
    "            f = r*p*2/(r+p)\n",
    "            print 'F1---%s'%f\n",
    "        if not np.isnan(f):\n",
    "            s += (float(sum(df['subsidy_x'] == i))/df.shape[0])*f\n",
    "    \n",
    "    print '\\nF1-macro---%s'%s\n",
    "    return s\n",
    "\n",
    "def f1_macro0_drop(label_truth, predictions):\n",
    "    df=pd.DataFrame(columns=[\"subsidy_x\",\"subsidy_y\"])\n",
    "    df.subsidy_y=predictions\n",
    "    df.subsidy_x=np.array(label_truth)\n",
    "    df.subsidy_y = df.subsidy_y.apply(lambda x:int(x))\n",
    "\n",
    "    \n",
    "    correct = df[df['subsidy_x'] == df['subsidy_y']]\n",
    "    s = 0\n",
    "    for i in [1000, 1500, 2000]:\n",
    "        if sum(df['subsidy_x'] == i)!=0:\n",
    "            r = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_x'] == i)\n",
    "        else: \n",
    "            r=0\n",
    "        if sum(df['subsidy_y'] == i)!=0:\n",
    "            p = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_y'] == i)        \n",
    "        else:\n",
    "            p=0\n",
    "            \n",
    "        if (r+p)!=0:\n",
    "            f = r*p*2/(r+p)\n",
    "        else:\n",
    "            f=0\n",
    "        if not np.isnan(f):\n",
    "            s += (float(sum(df['subsidy_x'] == i))/df.shape[0])*f\n",
    "    \n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nice_feature=pd.read_csv('input/nice_feature.csv',header=None,index_col=0)\n",
    "feature_imp_place20=pd.read_csv('input/feature_imp_place20.csv')\n",
    "#big_nice_feature=pd.read_csv('input/big_nice_feature.csv')\n",
    "\n",
    "target = 'label'\n",
    "IDcol = 'id'\n",
    "\n",
    "all_feature = [x for x in train_test.columns if x not in [target,IDcol]]\n",
    "\n",
    "predictors=all_feature\n",
    "#predictors = [ x for x in all_feature if (x in nice_feature.index)|(x in feature_imp_place20.feature.values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = train_test.ix[:, train_test.columns != 'label']\n",
    "y = train_test.ix[:, train_test.columns == 'label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rng = np.random.RandomState(42)\n",
    "kf = KFold(y.shape[0], n_folds=3, shuffle=True,random_state=rng)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#skf=StratifiedKFold(z,n_folds=3,shuffle=True,random_state=rng)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def model_tuning():\n",
    "    \n",
    "#for tuning_target in [1]:\n",
    "    score=[]\n",
    "    for train_index, test_index in kf:\n",
    "        train,train['label']=X.iloc[train_index],y.iloc[train_index]\n",
    "    \n",
    "        Oversampling1000 = train.loc[train.label == 1000]\n",
    "        Oversampling1500 = train.loc[train.label == 1500]\n",
    "        Oversampling2000 = train.loc[train.label == 2000]\n",
    "        for i in range(a):\n",
    "            train = train.append(Oversampling1000)\n",
    "        for j in range(b):\n",
    "            train = train.append(Oversampling1500)\n",
    "        for k in range(c):\n",
    "            train = train.append(Oversampling2000)\n",
    "        \n",
    "        param_dist = {\n",
    "            'n_estimators': 120,\n",
    "            'max_depth': 7,\n",
    "            'min_child_weight':7,\n",
    "            'gamma':0,\n",
    "            'subsample':0.4,\n",
    "            'colsample_bytree':1,\n",
    "            'reg_alpha':0.1,\n",
    "            'learning_rate':0.05}\n",
    "        \n",
    "        '''\n",
    "          param_dist = {\n",
    "            'n_estimators': 60,\n",
    "            'max_depth': 5,\n",
    "            'learning_rate':0.2,\n",
    "            'min_child_weight':2,\n",
    "            'gamma':0,\n",
    "            'subsample':1,\n",
    "            'colsample_bytree':1,\n",
    "            'reg_alpha':0.008}\n",
    "        '''      \n",
    "        xgb_model = XGBClassifier(**param_dist).fit(train[predictors],train[target])\n",
    "        predict = xgb_model.predict(X.iloc[test_index][predictors])\n",
    "        \n",
    "        true = y.iloc[test_index]\n",
    "        print f1_macro0_drop(true,predict)\n",
    "        score.append(f1_macro0_drop(true,predict))\n",
    "    #print tuning_target,\n",
    "    print pd.DataFrame(score).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "param_dist = {\n",
    "    'n_estimators': 60,\n",
    "    'max_depth': 5,\n",
    "    'learning_rate':0.2,\n",
    "    'min_child_weight':2,\n",
    "    'gamma':0,\n",
    "    'subsample':1,\n",
    "    'colsample_bytree':1,\n",
    "    'reg_alpha':0.008}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_dist={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 7 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kuhung/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.026782462924\n",
      "0.0300717951332\n",
      "0.0231621896966\n",
      "0    0.026672\n",
      "dtype: float64\n",
      "7 7 10\n",
      "0.0241640922619\n",
      "0.0295010482095\n",
      "0.0231596239668\n",
      "0    0.025608\n",
      "dtype: float64\n",
      "7 7 11\n",
      "0.0211143931036\n",
      "0.0255566571076\n",
      "0.0223383298463\n",
      "0    0.023003\n",
      "dtype: float64\n",
      "7 8 9\n",
      "0.0217725456119\n",
      "0.0269171827082\n",
      "0.0230998396852\n",
      "0    0.02393\n",
      "dtype: float64\n",
      "7 8 10\n",
      "0.0226186385422\n",
      "0.0283371925543\n",
      "0.0225196361587\n",
      "0    0.024492\n",
      "dtype: float64\n",
      "7 8 11\n",
      "0.0263329042129\n",
      "0.0281134383289\n",
      "0.0242123380531\n",
      "0    0.02622\n",
      "dtype: float64\n",
      "7 10 9\n",
      "0.026892937461\n",
      "0.0295451530542\n",
      "0.0244650841901\n",
      "0    0.026968\n",
      "dtype: float64\n",
      "7 10 10\n",
      "0.0253996823388\n",
      "0.0291622706251\n",
      "0.0218234318643\n",
      "0    0.025462\n",
      "dtype: float64\n",
      "7 10 11\n",
      "0.0253800368276\n",
      "0.0279395940109\n",
      "0.0226335057296\n",
      "0    0.025318\n",
      "dtype: float64\n",
      "8 7 9\n",
      "0.0254632302751\n",
      "0.0290193586694\n",
      "0.0230141449743\n",
      "0    0.025832\n",
      "dtype: float64\n",
      "8 7 10\n",
      "0.0238592692349\n",
      "0.0308250561484\n",
      "0.0247153348616\n",
      "0    0.026467\n",
      "dtype: float64\n",
      "8 7 11\n",
      "0.0246709572551\n",
      "0.0298414449095\n",
      "0.0242591872245\n",
      "0    0.026257\n",
      "dtype: float64\n",
      "8 8 9\n",
      "0.0249674174468\n",
      "0.0286258273442\n",
      "0.0213301229079\n",
      "0    0.024974\n",
      "dtype: float64\n",
      "8 8 10\n",
      "0.0257317775683\n",
      "0.0322416936775\n",
      "0.0221057432742\n",
      "0    0.026693\n",
      "dtype: float64\n",
      "8 8 11\n",
      "0.0244809671697\n",
      "0.0312608562974\n",
      "0.0233848594192\n",
      "0    0.026376\n",
      "dtype: float64\n",
      "8 10 9\n",
      "0.0255446993128\n",
      "0.0293815289241\n",
      "0.0225920494413\n",
      "0    0.025839\n",
      "dtype: float64\n",
      "8 10 10\n",
      "0.0243757403148\n",
      "0.0310548904191\n",
      "0.021272039764\n",
      "0    0.025568\n",
      "dtype: float64\n",
      "8 10 11\n",
      "0.0251797375455\n",
      "0.0294568626805\n",
      "0.0242094816802\n",
      "0    0.026282\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#大于线上成绩的模型都可采用，下面的参数都可并入集成子模型\n",
    "for a in [7,8]:\n",
    "    for b in [7,8,10]:\n",
    "        for c in [9,10,11]:\n",
    "            print a,b,c\n",
    "            model_tuning()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "0.0310752036941\n",
    "0.0318844076325\n",
    "0.0312565245932\n",
    "0    0.031405\n",
    "dtype: float64\n",
    "7 7 10\n",
    "0.0306199421948\n",
    "0.0300459478888\n",
    "0.0305127222103\n",
    "0    0.030393\n",
    "dtype: float64\n",
    "7 7 11\n",
    "0.0307264298572\n",
    "0.0293045287458\n",
    "0.0312028275973\n",
    "0    0.030411\n",
    "dtype: float64\n",
    "7 8 9\n",
    "0.0316870281635\n",
    "0.0324066603627\n",
    "0.0319350212627\n",
    "0    0.03201\n",
    "dtype: float64\n",
    "7 8 10\n",
    "0.0321757374537\n",
    "0.0308081674416\n",
    "0.0309890352377\n",
    "0    0.031324\n",
    "dtype: float64\n",
    "7 8 11\n",
    "0.0317321625774\n",
    "0.0320268365579\n",
    "0.0320917017139\n",
    "0    0.03195\n",
    "dtype: float64\n",
    "7 10 9\n",
    "0.0307964734137\n",
    "0.0339801180803\n",
    "0.0337253421053\n",
    "0    0.032834\n",
    "dtype: float64\n",
    "7 10 10\n",
    "0.0295198193324\n",
    "0.0320932018262\n",
    "0.0302967463066\n",
    "0    0.030637\n",
    "dtype: float64\n",
    "7 10 11\n",
    "0.0303200639973\n",
    "0.0337892702597\n",
    "0.0343904289977\n",
    "0    0.032833\n",
    "dtype: float64\n",
    "8 7 9\n",
    "0.0297409167768\n",
    "0.0301859502139\n",
    "0.0302600182185\n",
    "0    0.030062\n",
    "dtype: float64\n",
    "8 7 10\n",
    "0.0292952632517\n",
    "0.0334291177054\n",
    "0.0317325394815\n",
    "0    0.031486\n",
    "dtype: float64\n",
    "8 7 11\n",
    "0.0318542646738\n",
    "0.0312225427818\n",
    "0.028182098376\n",
    "0    0.03042\n",
    "dtype: float64\n",
    "8 8 9\n",
    "0.031904806337\n",
    "0.0339781548937\n",
    "0.0318626459528\n",
    "0    0.032582\n",
    "dtype: float64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "0.03+: 555 579 599 779 799 977 979 999\n",
    "\n",
    "0.031+: 579 588 688 689 699 768 779 789 878 897\n",
    "\n",
    "all 0.03: 588 599 689 778 779 7810 7109 71010 878 8710 888 81010\n",
    "    \n",
    "all 0.031: 588 689 779 7810 7109 71010 71011 878 8710 81010\n",
    "    \n",
    "\n",
    "8710\n",
    "\n",
    "7109"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def model_tuning_GBM():\n",
    "    \n",
    "#for tuning_target in [1]:\n",
    "    score=[]\n",
    "    for train_index, test_index in kf:\n",
    "        train,train['label']=X.iloc[train_index],y.iloc[train_index]\n",
    "    \n",
    "        Oversampling1000 = train.loc[train.label == 1000]\n",
    "        Oversampling1500 = train.loc[train.label == 1500]\n",
    "        Oversampling2000 = train.loc[train.label == 2000]\n",
    "        for i in range(a):\n",
    "            train = train.append(Oversampling1000)\n",
    "        for j in range(b):\n",
    "            train = train.append(Oversampling1500)\n",
    "        for k in range(c):\n",
    "            train = train.append(Oversampling2000)\n",
    "        \n",
    "\n",
    "        param_dist = {\n",
    "            'n_estimators': 60,\n",
    "            'learning_rate':0.1,\n",
    "            'max_depth': 5,\n",
    "            'min_samples_split':200\n",
    "            'min_samples_leaf':30\n",
    "        }\n",
    "\n",
    "            \n",
    "        gb_model = GradientBoostingClassifier(**param_dist).fit(train[predictors],train[target])\n",
    "        predict = gb_model.predict(X.iloc[test_index][predictors])\n",
    "        true = y.iloc[test_index]\n",
    "        print f1_macro0_drop(true,predict)\n",
    "        score.append(f1_macro0_drop(true,predict))\n",
    "    #print tuning_target,\n",
    "    print pd.DataFrame(score).mean()\n",
    "    \n",
    "    #基本款0.031609"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kuhung/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0296351871405\n",
      "0.0344766293693\n",
      "0.0317018726146\n",
      "0    0.031938\n",
      "dtype: float64\n",
      "20 140\n",
      "0.0308879844468\n",
      "0.0317513333715\n",
      "0.0304036847572\n",
      "0    0.031014\n",
      "dtype: float64\n",
      "20 160\n",
      "0.0298885533991\n",
      "0.0346910077952\n",
      "0.028275184921\n",
      "0    0.030952\n",
      "dtype: float64\n",
      "20 180\n",
      "0.0317100176381\n",
      "0.0329440129865\n",
      "0.0305723948037\n",
      "0    0.031742\n",
      "dtype: float64\n",
      "23 120\n",
      "0.0318638744139\n",
      "0.033556942373\n",
      "0.0270590851154\n",
      "0    0.030827\n",
      "dtype: float64\n",
      "23 140\n",
      "0.0304454328629\n",
      "0.0332689674271\n",
      "0.0291043926546\n",
      "0    0.03094\n",
      "dtype: float64\n",
      "23 160\n",
      "0.029764275608\n",
      "0.0339638609346\n",
      "0.0272915911394\n",
      "0    0.03034\n",
      "dtype: float64\n",
      "23 180\n",
      "0.0304404014181\n",
      "0.0342046445681\n",
      "0.0280626566921\n",
      "0    0.030903\n",
      "dtype: float64\n",
      "26 120\n",
      "0.032038297279\n",
      "0.0334043060478\n",
      "0.0280919800046\n",
      "0    0.031178\n",
      "dtype: float64\n",
      "26 140\n",
      "0.0310400654482\n",
      "0.0339083019306\n",
      "0.0263186111739\n",
      "0    0.030422\n",
      "dtype: float64\n",
      "26 160\n",
      "0.030533029595\n",
      "0.0338980157542\n",
      "0.0305160483516\n",
      "0    0.031649\n",
      "dtype: float64\n",
      "26 180\n",
      "0.0308518298194\n",
      "0.0350676058903\n",
      "0.02735735858\n",
      "0    0.031092\n",
      "dtype: float64\n",
      "29 120\n",
      "0.0309348305318\n",
      "0.0331186743853\n",
      "0.0294012553724\n",
      "0    0.031152\n",
      "dtype: float64\n",
      "29 140\n",
      "0.029479726025\n",
      "0.0351899631982\n",
      "0.0274822416253\n",
      "0    0.030717\n",
      "dtype: float64\n",
      "29 160\n",
      "0.03150068603\n",
      "0.0344991621963\n",
      "0.0304905955186\n",
      "0    0.032163\n",
      "dtype: float64\n",
      "29 180\n",
      "0.0309975845205\n",
      "0.0352478330155\n",
      "0.0294395172891\n",
      "0    0.031895\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "for min_samples_leaf in range(20,31,3):\n",
    "    for min_samples_split in range(120,181,20):\n",
    "        print min_samples_leaf,min_samples_split\n",
    "        model_tuning_GBM()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size = 0.3, random_state = 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kuhung/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  from ipykernel import kernelapp as app\n",
      "/home/kuhung/anaconda2/lib/python2.7/site-packages/ipykernel/__main__.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "train=X_train\n",
    "train['label']=y_train\n",
    "\n",
    "test=X_test\n",
    "test['label']=y_test\n",
    "\n",
    "test.to_csv('input/test_7r07part.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nice_feature=pd.read_csv('input/nice_feature.csv',header=None,index_col=0)\n",
    "\n",
    "target = 'label'\n",
    "IDcol = 'id'\n",
    "ids = test['id'].values\n",
    "\n",
    "all_feature = [x for x in train.columns if x not in [target]]\n",
    "\n",
    "feature_imp_place20=pd.read_csv('input/feature_imp_place20.csv')\n",
    "predictors = [ x for x in all_feature if (x in nice_feature.index)|(x in feature_imp_place20.feature.values)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0       6530\n",
       "1000.0     515\n",
       "1500.0     335\n",
       "2000.0     239\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Number of normal transactions: ', 1089)\n",
      "('Number of fraud transactions: ', 1089)\n",
      "('Total number of transactions in resampled data: ', 2178)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# tips：\n",
    "# 尝试欠采样和smoth采样\n",
    "\n",
    "# undersample\n",
    "train.to_csv('input/train_7r07part.csv',index=False)\n",
    "train=pd.read_csv('input/train_7r07part.csv')\n",
    "\n",
    "number_records_fraud = len(train[train.label != 0])\n",
    "fraud_indices = np.array(train[train.label != 0].index)\n",
    "\n",
    "# Picking the indices of the normal classes\n",
    "normal_indices = train[train.label == 0].index\n",
    "\n",
    "# Out of the indices we picked, randomly select \"x\" number (number_records_fraud)\n",
    "random_normal_indices = np.random.choice(normal_indices, number_records_fraud, replace = False)\n",
    "random_normal_indices = np.array(random_normal_indices)\n",
    "\n",
    "# Appending the 2 indices\n",
    "under_sample_indices = np.concatenate([fraud_indices,random_normal_indices])\n",
    "\n",
    "# Under sample dataset\n",
    "under_sample_data = train.iloc[under_sample_indices,:]\n",
    "\n",
    "X_undersample = under_sample_data.ix[:, under_sample_data.columns != 'label']\n",
    "y_undersample = under_sample_data.ix[:, under_sample_data.columns == 'label']\n",
    "\n",
    "# Showing ratio\n",
    "print(\"Number of normal transactions: \", len(under_sample_data[under_sample_data.label == 0]))\n",
    "print(\"Number of fraud transactions: \", len(under_sample_data[under_sample_data.label != 0]))\n",
    "print(\"Total number of transactions in resampled data: \", len(under_sample_data))\n",
    "\n",
    "\n",
    "# Oversample\n",
    "Oversampling1000 = train.loc[train.label == 1000]\n",
    "Oversampling1500 = train.loc[train.label == 1500]\n",
    "Oversampling2000 = train.loc[train.label == 2000]\n",
    "'''\n",
    "for i in range(6):\n",
    "    train = train.append(Oversampling1000)\n",
    "for j in range(10):\n",
    "    train = train.append(Oversampling1500)\n",
    "for k in range(12):\n",
    "    train = train.append(Oversampling2000)\n",
    "'''\n",
    "for i in range(5):\n",
    "    train = train.append(Oversampling1000)\n",
    "for j in range(8):\n",
    "    train = train.append(Oversampling1500)\n",
    "for k in range(10):\n",
    "    train = train.append(Oversampling2000)\n",
    "    \n",
    "train = train[train['label'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15264, 432)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "r_scale=preprocessing.RobustScaler()\n",
    "\n",
    "train_scaled=r_scale.fit_transform(train[predictors])\n",
    "test_scaled=r_scale.transform(test[predictors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.031658650391196586"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf=XGBClassifier(max_depth=4,objective='multi:softmax',n_estimators=100,seed=42)\n",
    "clf = clf.fit(train[predictors],train[target])\n",
    "result = clf.predict(test[predictors])\n",
    "\n",
    "f1_macro(y_test,result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def f1_macro0_drop(label_truth, predictions):\n",
    "    df=pd.DataFrame(columns=[\"subsidy_x\",\"subsidy_y\"])\n",
    "    df.subsidy_y=predictions\n",
    "    df.subsidy_x=np.array(label_truth)\n",
    "    df.subsidy_y = df.subsidy_y.apply(lambda x:int(x))\n",
    "\n",
    "    \n",
    "    correct = df[df['subsidy_x'] == df['subsidy_y']]\n",
    "    s = 0\n",
    "    for i in [1000, 1500, 2000]:\n",
    "        if sum(df['subsidy_x'] == i)!=0:\n",
    "            r = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_x'] == i)\n",
    "        else: \n",
    "            r=0\n",
    "        if sum(df['subsidy_y'] == i)!=0:\n",
    "            p = float(sum(correct['subsidy_y'] == i))/sum(df['subsidy_y'] == i)        \n",
    "        else:\n",
    "            p=0\n",
    "            \n",
    "        if (r+p)!=0:\n",
    "            f = r*p*2/(r+p)\n",
    "        else:\n",
    "            f=0\n",
    "        if not np.isnan(f):\n",
    "            s += (float(sum(df['subsidy_x'] == i))/df.shape[0])*f\n",
    "    \n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "my_score=make_scorer(score_func=f1_macro0_drop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 参数调节\n",
    "\n",
    "from sklearn import cross_validation, metrics  \n",
    "from sklearn.model_selection import GridSearchCV \n",
    "\n",
    "def modelfit(alg, dtrain, predictors, target,performCV=True, printFeatureImportance=False, cv_folds=3):\n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(dtrain[predictors], dtrain[target])\n",
    "\n",
    "    #Predict training set:\n",
    "    dtrain_predictions = alg.predict(dtrain[predictors])\n",
    "    dtrain_predprob = alg.predict_proba(dtrain[predictors])[:,1]\n",
    "\n",
    "    #Perform cross-validation:\n",
    "    if performCV:\n",
    "        cv_score = cross_validation.cross_val_score(alg, dtrain[predictors], dtrain[target], cv=cv_folds, scoring=my_score)\n",
    "\n",
    "    #Print model report:\n",
    "    print \"\\nModel Report\"\n",
    "    print \"Accuracy : %.4g\" % metrics.accuracy_score(dtrain[target].values, dtrain_predictions)\n",
    "    \n",
    "    if performCV:\n",
    "        print \"CV Score : Mean - %.7g | Std - %.7g | Min - %.7g | Max - %.7g\" % (np.mean(cv_score),np.std(cv_score),np.min(cv_score),np.max(cv_score))\n",
    "\n",
    "    #Print Feature Importance:\n",
    "    if printFeatureImportance:\n",
    "        feat_imp = pd.Series(alg.feature_importances_, predictors).sort_values(ascending=False)\n",
    "        feat_imp.plot(kind='bar', title='Feature Importances')\n",
    "        plt.ylabel('Feature Importance Score')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.8563\n",
      "CV Score : Mean - 0.1463062 | Std - 0.01041427 | Min - 0.1318068 | Max - 0.1557945\n"
     ]
    }
   ],
   "source": [
    "gbm0 = GradientBoostingClassifier(random_state=10)\n",
    "modelfit(gbm0, under_sample_data, predictors,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import make_pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def Oversampling(train):\n",
    "    # Oversample\n",
    "    Oversampling1000 = train.loc[train.label == 1000]\n",
    "    Oversampling1500 = train.loc[train.label == 1500]\n",
    "    Oversampling2000 = train.loc[train.label == 2000]\n",
    "\n",
    "    for i in range(5):\n",
    "        train = train.append(Oversampling1000)\n",
    "    for j in range(8):\n",
    "        train = train.append(Oversampling1500)\n",
    "    for k in range(10):\n",
    "        train = train.append(Oversampling2000)\n",
    "    \n",
    "    train = train[train['label'].notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kuhung/anaconda2/lib/python2.7/site-packages/sklearn/model_selection/_search.py:662: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: 0.12288, std: 0.01573, params: {'n_estimators': 20},\n",
       "  mean: 0.12506, std: 0.00925, params: {'n_estimators': 30},\n",
       "  mean: 0.13755, std: 0.01015, params: {'n_estimators': 40},\n",
       "  mean: 0.13984, std: 0.01478, params: {'n_estimators': 50},\n",
       "  mean: 0.13868, std: 0.01559, params: {'n_estimators': 60},\n",
       "  mean: 0.14296, std: 0.01391, params: {'n_estimators': 70},\n",
       "  mean: 0.14423, std: 0.01627, params: {'n_estimators': 80}],\n",
       " {'n_estimators': 80},\n",
       " 0.14422876456755157)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "param_test1 = {'n_estimators':range(20,81,10)}\n",
    "gsearch1 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, min_samples_split=80,min_samples_leaf=8,max_depth=3,max_features='sqrt',subsample=0.8,random_state=10), \n",
    "param_grid = param_test1, scoring=my_score,n_jobs=4,iid=False, cv=5)\n",
    "#gsearch1.fit(train[predictors],train[target])\n",
    "\n",
    "gsearch1.fit(X_undersample[predictors],y_undersample[target])\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.8655\n",
      "CV Score : Mean - 0.001894665 | Std - 0.00378933 | Min - 0 | Max - 0.009473324\n"
     ]
    }
   ],
   "source": [
    "xgb0 = XGBClassifier(max_depth=4,objective='multi:softmax',n_estimators=100,seed=42,base_score=2000)\n",
    "modelfit(xgb0, train_test, predictors,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.985\n",
      "CV Score : Mean - 0.002591079 | Std - 0.004933227 | Min - 0 | Max - 0.01244976\n"
     ]
    }
   ],
   "source": [
    "rf0=RandomForestClassifier()\n",
    "modelfit(rf0, train_test, predictors,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Model Report\n",
      "Accuracy : 0.987\n",
      "CV Score : Mean - 0.003399415 | Std - 0.005718429 | Min - 0 | Max - 0.01478254\n"
     ]
    }
   ],
   "source": [
    "rf0=RandomForestClassifier(class_weight='balanced')\n",
    "modelfit(rf0, train_test, predictors,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## why the score of it is higher?\n",
    "## Because of resample.After resample,the dis of dataset had changed.\n",
    "# 0.02779836483035414\n",
    "\n",
    "xgb0 = XGBClassifier(max_depth=4,objective='multi:softmax',n_estimators=100,seed=42)\n",
    "modelfit(xgb0, under_sample_data, predictors,target)\n",
    "\n",
    "param_test1 = {'n_estimators':range(20,81,10)}\n",
    "gsearch1 = GridSearchCV(estimator = GradientBoostingClassifier(learning_rate=0.1, min_samples_split=80,min_samples_leaf=8,max_depth=3,max_features='sqrt',subsample=0.8,random_state=10), \n",
    "param_grid = param_test1, scoring=my_score,n_jobs=4,iid=False, cv=5)\n",
    "#gsearch1.fit(train[predictors],train[target])\n",
    "gsearch1.fit(X_undersample,y_undersample)\n",
    "\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "## Feature importance\n",
    "'''\n",
    "clf=XGBClassifier(max_depth=4,objective='multi:softmax',n_estimators=100,seed=42)\n",
    "clf = clf.fit(train[predictors],train[target])\n",
    "result = clf.predict_proba(test[predictors])\n",
    "f1_macro(y_test,result)\n",
    "\n",
    "feat_imp=pd.Series(clf.booster().get_fscore()).sort_values(ascending=False)\n",
    "feat_imp.plot(kind='bar', title='Feature Importances')\n",
    "feat_imp.tail(50)\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''\n",
    "## Evaluation\n",
    "def score(df):\n",
    "    # df有三列，ID:学生ID,subsidy_x:实际奖学金金额,subsidy_y:预测奖学金金额\n",
    "    correct = test_result[test_result['subsidy_x'] == test_result['subsidy_y']]\n",
    "    s = 0\n",
    "    for i in [1000, 1500, 2000]:\n",
    "        r = float(sum(correct['subsidy_y'] == i))/sum(test_result['subsidy_x'] == i)\n",
    "        p = float(sum(correct['subsidy_y'] == i))/sum(test_result['subsidy_y'] == i)\n",
    "        f = r*p*2/(r+p)\n",
    "        if not np.isnan(f):\n",
    "            s += (float(sum(test_result['subsidy_x'] == i))/test_result.shape[0])*f\n",
    "    print(s)\n",
    "\n",
    "test_result = pd.DataFrame(columns=[\"studentid\",\"subsidy_x\",\"subsidy_y\"])\n",
    "test_result.studentid = ids\n",
    "\n",
    "test_result.subsidy_x =np.array(y_test)\n",
    "test_result.subsidy_y = result\n",
    "test_result.subsidy_y = test_result.subsidy_y.apply(lambda x:int(x))\n",
    "\n",
    "score(test_result)\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
